﻿using System;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using Unikado.PHPLibs.Tokenizer.Types;

namespace Unikado.PHPLibs.Tokenizer
{

    internal class PhpCodeStreamParser
    {

        #region - - - - - -   P R I V A T E   F I E L D S   - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        private StreamReader reader;
        private readonly TokenCollection result;

        #region // temporary fields for parsing

        private bool isPhp;
        private bool isStrOpen;
        private bool isCommentOpen;
        private char strOpenChar;
        private bool isHnDocOpen;
        private string hnDocID;
        private StringBuilder sb;
        private int escapes;
        private char c0;
        private int cn;
        private char c1;
        private int lineNumber;
        private int bLevel;
        private int cLevel;
        private int sLevel;
        private bool afterLF;

        #endregion

        #endregion

        #region - - - - - -   C O N S T R U C T O R S   - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        public PhpCodeStreamParser()
        {
            result = new TokenCollection();
        }

        #endregion

        #region - - - - - -   P U B L I C   M E T H O D S   - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        public TokenCollection Parse(Stream phpCodeStream, Encoding encoding)
        {
            initFields();
            using (reader = new StreamReader(phpCodeStream, encoding))
            {
                while (reader.Peek() > -1)
                {
                    readNext();
                    if ('\r' == c0) continue;                                  // Ignore all CRs (\r)
                    if (!isPhp) { findPhpOpenTag(); continue; }                // Find next PHP open tag by need
                    if (handleOpenedComment()) continue;                       // Handle a currently opened Multiline comment or DocBlock comment.
                    if (handleOpenedString()) continue;                        // Handle a currently opened string (in single or double quotes
                    if (handleOpenedHnDoc()) continue;                         // Handle a currently opened HEREDOC or NOWDOC string
                    if (handlePhpCloseTag()) continue;                         // Handle a PHP close tag ?>
                    if (handleSpaces()) continue;                              // Handle spaces + tabulators
                    if (handleNewLine()) continue;                             // Handle new lines (\n)
                    bool isWordChar = ParseHelper.IsWordStartChar(c0);
                    TypeFine fType;
                    TypeNormal nType;
                    TypeRough rType;
                    var last = result.LastToken;
                    bool isDeclaration = last != null && ParseHelper.IsDefinition(last.Code);
                    if (!isWordChar)
                    {
                        if (handleBrackets()) continue;                        // {  [  (  )  ]  }
                        if (handleEqualSign()) continue;                       // =  ==  ===  =>
                        if (handleNot()) continue;                             // !  !=  !==
                        if (handleAnd()) continue;                             // &  &&  &=
                        if (handleOr()) continue;                              // |  ||  |=
                        if (handleSlash()) continue;                           // /  /=  //  /*
                        if (handleComSem()) continue;                          // ;  ,  @
                        if (handleDot()) continue;                             // .  .=
                        if (handleLeftArrows()) continue;                      // <  <<  <>  <=  <<=  <<<'NOWDOC'  <<<HEREDOC
                        if (handleRightArrows()) continue;                     // >  >>  >=  >>=
                        if (handlePlus()) continue;                            // +  ++  +=
                        if (handleMinus()) continue;                           // -  --  -=
                        if (handleVars())
                        {
                            if (TypeHelper.ConvertToFine(result.LastToken.Code, false, out fType))
                            {
                                nType = TypeHelper.ConvertFineToNormal(fType);
                                rType = TypeHelper.ConvertNormalToRough(nType);
                                result.LastToken.FineType = fType;
                                result.LastToken.NormalType = nType;
                                result.LastToken.RoughType = rType;
                            }
                            continue;                            // $…  ${…}
                        }
                        if (handleStrOpen()) continue;                         // '…  "…  `…
                        if (handleHashComment()) continue;                     // #…
                        if (handleStarPerc()) continue;                        // *  *=  %  %=
                        if (handleColon()) continue;                           // :  ::
                        if (handleOtherOperators()) continue;                  // ?  ^  ^=
                        if (handleNumbers()) continue;                         // Handle some numerical (float or integer) definitions
                        throw new IOException("Parse error: Use of a unknown/unspecified PHP code character '" + c0 + "'!");
                    }
                    // a word begins
                    sb.Clear();
                    sb.Append(c0);
                    while (cn > -1 && ParseHelper.IsWordChar(c1)) { readNext(); sb.Append(c0); }
                    string str = sb.ToString();
                    sb.Clear();
                    if (TypeHelper.ConvertToFine(str, true, out fType, isDeclaration))
                    {
                        nType = TypeHelper.ConvertFineToNormal(fType);
                        rType = TypeHelper.ConvertNormalToRough(nType);
                        var t = result.Add(new Token(str, rType, nType, fType, "", lineNumber, bLevel, cLevel, sLevel));
                        lineNumber += t.LinebreakCount;
                        continue;
                    }
                    result.Add(new Token(str, TypeRough.Other, TypeNormal.Word, TypeFine.Word, "", lineNumber, bLevel, cLevel, sLevel));
                }
            }
            return result;
        }

        #endregion

        #region - - - - - -   P R I V A T E   M E T H O D S   - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        private void initFields()
        {
            isPhp = isCommentOpen = isStrOpen = isHnDocOpen = false;
            strOpenChar = '\0';
            hnDocID = null;
            sb = new StringBuilder();
            escapes = 0;
            result.Clear();
            lineNumber = 1;
            bLevel = 0;
            cLevel = 0;
            sLevel = 0;
            afterLF = false;
        }
        private bool readNext()
        {
            if (reader.Peek() < 0) return false;
            c0 = (char)reader.Read();
            cn = reader.Peek();
            c1 = cn < 0 ? '\0' : (char)cn;
            return cn > -1;
        }
        private string readCached(int count)
        {
            string res = "";
            int resCount = 0;
            for (int i = 0; i < count; i++)
            {
                if (reader.Peek() < 0) break;
                resCount++;
                res += (char)reader.Read();
            }
            reader.BaseStream.Position -= resCount;
            return res;
        }
        private void findPhpOpenTag()
        {
            bool doLoop = true;
            Token tok;
            while (doLoop)
            {
                while ('<' != c0 || ('?' != c1 && '%' != c1))
                {   // search for next <? or <%
                    sb.Append(c0);
                    if (cn < 0) break;
                    readNext();
                }
                if ('<' != c0)
                {   // No <? or <% was not found (end of file is reached)
                    tok = result.AddNoPhp(sb.ToString(), lineNumber, bLevel, cLevel, sLevel);
                    sb.Clear();
                    lineNumber += tok.LinebreakCount;
                    doLoop = false;
                    continue;
                }
                // <? or <% was found but if <?xml etc. is used, its not a php tag
                if ('%' == c1)
                {   // <% is for sure a php open tag
                    if (sb.Length > 0)
                    {
                        tok = result.AddNoPhp(sb.ToString(), lineNumber, bLevel, cLevel, sLevel);
                        lineNumber += tok.LinebreakCount;
                        sb.Clear();
                    }
                    result.AddPhpOpenTag("<%", lineNumber, bLevel, cLevel, sLevel);
                    readNext();
                    isPhp = true;
                    doLoop = false;
                    continue;
                }
                // <?
                if (!readNext())
                {   // file ends with <?
                    if (sb.Length > 0)
                    {
                        tok = result.AddNoPhp(sb.ToString(), lineNumber, bLevel, cLevel, sLevel);
                        lineNumber += tok.LinebreakCount;
                        sb.Clear();
                    }
                    result.AddPhpOpenTag("<?", lineNumber, bLevel, cLevel, sLevel);
                    isPhp = true;
                    doLoop = false;
                    continue;
                }
                if ('=' == c1)
                {
                    if (sb.Length > 0)
                    {
                        tok = result.AddNoPhp(sb.ToString(), lineNumber, bLevel, cLevel, sLevel);
                        lineNumber += tok.LinebreakCount;
                        sb.Clear();
                    }
                    result.AddPhpOpenTag("<?=", lineNumber, bLevel, cLevel, sLevel);
                    readNext();
                    isPhp = true;
                    doLoop = false;
                    continue;
                }
                if ('p' != char.ToLowerInvariant(c1))
                {
                    if (!ParseHelper.IsSimpleWordStartChar(c1))
                    {
                        result.AddPhpOpenTag("<?", lineNumber, bLevel, cLevel, sLevel);
                        isPhp = true;
                        doLoop = false;
                        continue;
                    }
                    sb.Append("<?");
                    readNext();
                    continue;
                }
                string s = readCached(3);
                if (s.Equals("php", StringComparison.InvariantCultureIgnoreCase))
                {
                    if (sb.Length > 0)
                    {
                        tok = result.AddNoPhp(sb.ToString(), lineNumber, bLevel, cLevel, sLevel);
                        lineNumber += tok.LinebreakCount;
                        sb.Clear();
                    }
                    result.AddPhpOpenTag("<?" + s, lineNumber, bLevel, cLevel, sLevel);
                    isPhp = true;
                    doLoop = false;
                    continue;
                }
                sb.Append("<?" + s);
            }
            if (sb.Length > 0)
            {
                tok = result.AddNoPhp(sb.ToString(), lineNumber, bLevel, cLevel, sLevel);
                lineNumber += tok.LinebreakCount;
                sb.Clear();
            }
        }
        private bool handleOpenedComment()
        {
            if (!isCommentOpen) return false;
            if ('*' != c0) { sb.Append(c0); return true; }
            if (cn < 0)
            {   // * is file end
                sb.Append(c0);
                var t = result.AddComment(sb.ToString(), TypeNormal.Comment, TypeFine.MultilineComment, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                lineNumber += t.LinebreakCount;
                sb.Clear();
                return true;
            }
            if ('/' != c1) { sb.Append(c0); return true; }
            readNext();
            sb.Append("*/");
            isCommentOpen = false;
            var tok = result.AddComment(sb.ToString(), lineNumber, bLevel, cLevel, sLevel);
            tok.IsAfterLinebreak = afterLF;
            afterLF = false;
            lineNumber += tok.LinebreakCount;
            sb.Clear();
            return true;
        }
        private bool handleOpenedString()
        {
            if (!isStrOpen) return false;
            while (isStrOpen)
            {
                while (strOpenChar != c0)
                {
                    if ('\\' == c0) escapes++;
                    else escapes = 0;
                    if ('\r' != c0) sb.Append(c0);
                    if (cn < 0) // end of file is reached without closing the string
                        throw new IOException("Invalid PHP code: It can not end with a unclosed string!");
                    readNext();
                }
                if (strOpenChar != c0) // end of file is reached without closing the string
                    throw new IOException("Invalid PHP code: It can not end with a unclosed string!");
                sb.Append(c0);
                if (0 == escapes || 0 == (escapes % 2))
                {   // this is the ending quote
                    var tok = result.AddString(sb.ToString(), lineNumber, bLevel, cLevel, sLevel);
                    tok.IsAfterLinebreak = afterLF;
                    afterLF = false;
                    lineNumber += tok.LinebreakCount;
                    sb.Clear();
                    isStrOpen = false;
                }
            }
            return true;
        }
        private bool handleOpenedHnDoc()
        {
            if (!isHnDocOpen || string.IsNullOrEmpty(hnDocID)) return false;
            while (isHnDocOpen)
            {
                while ('\n' != c0)
                {
                    if ('\r' != c0) sb.Append(c0);
                    if (cn < 0) // end of file is reached without closing the string
                        throw new IOException("Invalid PHP code: It can not end with a unclosed HEREDOC or NOWDOC string!");
                    readNext();
                }
                if (cn < 0) // end of file is reached without closing the string
                    throw new IOException("Invalid PHP code: It can not end with a unclosed HEREDOC or NOWDOC string!!");
                // c0 is \n
                string str = "";
                bool doBreak = false;
                for (int i = 0; cn > 0 && i < hnDocID.Length; i++)
                {
                    if (c1 != hnDocID[i])
                    {
                        sb.Append('\n').Append(str);
                        doBreak = true;
                        readNext();
                        break;
                    }
                    str += c1;
                    readNext();
                }
                if (doBreak) continue;
                // c1 is behind the ID
                if (!hnDocID.Equals(str, StringComparison.InvariantCulture))
                {   // Not the End
                    sb.Append('\n').Append(str);
                    readNext();
                    continue;
                }
                if (cn < 0 || ';' == c1 || ',' == c1 || ')' == c1)
                {
                    sb.Append('\n').Append(str);
                    var tok = result.AddString(sb.ToString(), lineNumber, bLevel, cLevel, sLevel);
                    tok.IsAfterLinebreak = afterLF;
                    afterLF = false;
                    lineNumber += tok.LinebreakCount;
                    isHnDocOpen = false;
                    continue;
                }
                sb.Append("\n" + str);
                readNext();
            }
            return true;
        }
        private bool handlePhpCloseTag()
        {
            if ('?' != c0 || '>' != c1) return false;
            var t = result.AddPhpCloseTag("?>", lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            readNext();
            isPhp = false;
            return true;
        }
        private bool handleSpaces()
        {
            if (' ' != c0 && '\t' != c0) return false;
            result.AddSpace(c0);
            return true;
        }
        private bool handleNewLine()
        {
            if ('\n' != c0) return false;
            result.AddLineBreak(lineNumber, bLevel, cLevel, sLevel);
            lineNumber++;
            afterLF = true;
            return true;
        }
        private bool handleBrackets()
        {
            if (!ParseHelper.IsBracket(c0)) return false;
            Token t;
            if ('{' == c0)
            {
                t = result.AddBracket(
                    "{", TypeNormal.OpenBracket, TypeFine.BlockOpenBracket, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                bLevel++;
            }
            else if ('(' == c0)
            {
                t = result.AddBracket(
                    "(", TypeNormal.OpenBracket, TypeFine.ConditionOpenBracket, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                cLevel++;
            }
            else if ('[' == c0)
            {
                t = result.AddBracket(
                    "[", TypeNormal.OpenBracket, TypeFine.SquareOpenBracket, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                sLevel++;
            }
            else if ('}' == c0)
            {
                t = result.AddBracket(
                    "}", TypeNormal.CloseBracket, TypeFine.BlockCloseBracket, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                bLevel--;
            }
            else if (')' == c0)
            {
                t = result.AddBracket(
                    ")", TypeNormal.CloseBracket, TypeFine.ConditionCloseBracket, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                cLevel--;
            }
            else
            {
                t = result.AddBracket(
                    "]", TypeNormal.CloseBracket, TypeFine.SquareCloseBracket, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                sLevel--;
            }
            return true;
        }
        private bool handleEqualSign() // =  ==  ===  =>
        {
            if ('=' != c0) return false;
            Token t;
            if (cn < 0)
            {
                t = result.AddAssignmentOperator("=", TypeFine.Assign, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('=' == c1)
            {   // ==
                if (!readNext())
                {
                    t = result.AddComparisonOperator("==", TypeFine.Equal, lineNumber, bLevel, cLevel, sLevel);
                    t.IsAfterLinebreak = afterLF;
                    afterLF = false;
                    return true;
                }
                if ('=' != c1)
                {
                    t = result.AddComparisonOperator("==", TypeFine.Equal, lineNumber, bLevel, cLevel, sLevel);
                    t.IsAfterLinebreak = afterLF;
                    afterLF = false;
                    return true;
                }
                readNext();
                t = result.AddComparisonOperator("===", TypeFine.EqualTypesafe, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('>' != c1)
            {
                t = result.AddAssignmentOperator("=", TypeFine.Assign, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            // =>
            readNext();
            t = result.AddAssignmentOperator("=>", TypeFine.AssignArray, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }
        private bool handleNot() // !  !=  !==
        {
            if ('!' != c0) return false;
            Token t;
            if (cn < 0)
            {
                t = result.AddLogicalOperator("!", TypeFine.LogicalBoolean, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('=' != c1)
            {
                t = result.AddLogicalOperator("!", TypeFine.LogicalBoolean, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            // !=
            if (!readNext())
            {
                t = result.AddComparisonOperator("!=", TypeFine.Equal, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('=' != c1)
            {
                t = result.AddComparisonOperator("!=", TypeFine.Equal, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            readNext();
            t = result.AddComparisonOperator("!==", TypeFine.EqualTypesafe, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }
        private bool handleAnd() // &  &=  &&
        {
            if ('&' != c0) return false;
            Token t;
            if (cn < 0)
            {
                t = result.AddOperator("&", TypeNormal.BitwiseOperator, TypeFine.BitwiseOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('&' == c1)
            {
                readNext();
                t = result.AddLogicalOperator("&&", TypeFine.LogicalBoolean, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('=' != c1)
            {
                t = result.AddOperator("&", TypeNormal.BitwiseOperator, TypeFine.BitwiseOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            readNext();
            t = result.AddAssignmentOperator("&=", TypeFine.AssignBitwise, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }
        private bool handleOr() // |  |=  ||
        {
            if ('|' != c0) return false;
            Token t;
            if (cn < 0)
            {
                t = result.AddOperator("|", TypeNormal.BitwiseOperator, TypeFine.BitwiseOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('|' == c1)
            {
                readNext();
                t = result.AddLogicalOperator("||", TypeFine.LogicalBoolean, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('=' != c1)
            {
                t = result.AddOperator("|", TypeNormal.BitwiseOperator, TypeFine.BitwiseOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            readNext();
            t = result.AddAssignmentOperator("|=", TypeFine.AssignBitwise, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }
        private bool handleSlash() // /  /=  //…
        {
            if ('/' != c0) return false;
            Token t;
            if (cn < 0)
            {
                t = result.AddAritmethicOperator("/", TypeFine.AritmethicOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('=' == c1)
            {
                readNext();
                t = result.AddAssignmentOperator("/=", TypeFine.AssignAritmethic, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('*' == c1)
            {
                readNext();
                sb.Clear();
                sb.Append("/*");
                isCommentOpen = true;
                while (isCommentOpen)
                {
                    while (cn > -1 && '*' != c1)
                    {
                        if ('\r' != c1) sb.Append(c1);
                        readNext();
                    }
                    if (cn < 0) throw new IOException("Parse error: PHP code can not end with a unclosed /*… comment");
                    readNext();
                    sb.Append(c0);
                    if (cn < 0) throw new IOException("Parse error: PHP code can not end with a unclosed /*… comment");
                    if ('/' != c1) continue;
                    isCommentOpen = false;
                    readNext();
                    sb.Append("/");
                    t = result.AddComment(sb.ToString(), lineNumber, bLevel, cLevel, sLevel);
                    t.IsAfterLinebreak = afterLF;
                    afterLF = false;
                    lineNumber += t.LinebreakCount;
                    sb.Clear();
                }
                return true;
            }
            if ('/' != c1)
            {
                t = result.AddAritmethicOperator("/", TypeFine.AritmethicOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            sb.Clear();
            sb.Append("//");
            readNext();
            while (cn > -1 && '\r' != c1 && '\n' != c1)
            {
                readNext();
                sb.Append(c0);
            }
            t = result.AddComment(sb.ToString(), TypeNormal.Comment, TypeFine.SlashComment, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            sb.Clear();
            return true;
        }
        private bool handleLeftArrows() // < << <= <<= <<< <>
        {
            if ('<' != c0) return false;
            Token t;
            if (cn < 0)
            {
                t = result.AddComparisonOperator("<", TypeFine.EqualNumerical, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('=' == c1)
            {
                readNext();
                t = result.AddComparisonOperator("<=", TypeFine.EqualNumerical, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('>' == c1)
            {
                readNext();
                t = result.AddComparisonOperator("<>", TypeFine.EqualNumerical, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('<' != c1)
            {
                t = result.AddComparisonOperator("<", TypeFine.EqualNumerical, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            readNext();
            // <<
            if (cn < 0)
            {
                t = result.AddOperator("<<", TypeNormal.BitwiseOperator, TypeFine.BitwiseShift, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('=' == c1)
            {
                readNext();
                t = result.AddAssignmentOperator("<<=", TypeFine.AssignBitwise, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('<' != c1)
            {
                t = result.AddOperator("<<", TypeNormal.BitwiseOperator, TypeFine.BitwiseShift, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            readNext();
            // <<<
            if (!ParseHelper.IsSimpleWordStartChar(c1) && '\'' != c1)
                throw new IOException("Parse error: <<< can only be used as a HEREDOC or NOWDOC initializer.");
            string str = "";
            if ('\'' == c1)
            {
                while (readNext() && '\'' != c1) str += c1;
                if (str.Length < 1 || !ParseHelper.IsSimpleWord(str))
                    throw new IOException("Parse error: <<<'XXX' can only be used as a NOWDOC initializer. (XXX must be a valid PHP word)");
                readNext(); // c0 is now '
                sb.Clear();
                sb.Append("<<<'").Append(str).Append('\'');
                hnDocID = str;
                isHnDocOpen = true;
                return true;
            }
            // ABC
            while (cn > -1 && ParseHelper.IsSimpleWordStartChar(c1)) { str += c1; readNext(); }
            sb.Clear();
            sb.Append("<<<").Append(str);
            hnDocID = str;
            isHnDocOpen = true;
            return true;
        }
        private bool handleRightArrows() // > >> >= >>=
        {
            if ('>' != c0) return false;
            Token t;
            if (cn < 0)
            {
                t = result.AddComparisonOperator(">", TypeFine.EqualNumerical, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('=' == c1)
            {
                readNext();
                t = result.AddComparisonOperator(">=", TypeFine.EqualNumerical, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('>' != c1)
            {
                t = result.AddComparisonOperator(">", TypeFine.EqualNumerical, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            readNext();
            // >>
            if (cn < 0 || '=' != c1)
            {
                t = result.AddOperator(">>", TypeNormal.BitwiseOperator, TypeFine.BitwiseShift, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            readNext();
            t = result.AddAssignmentOperator(">>=", TypeFine.AssignBitwise, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }
        private bool handlePlus() // + ++ +=
        {
            if ('+' != c0) return false;
            Token t;
            if (cn < 0)
            {
                t = result.AddAritmethicOperator("+", TypeFine.AritmethicOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('=' == c1)
            {
                readNext();
                t = result.AddAssignmentOperator("+=", TypeFine.AssignAritmethic, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('+' != c1)
            {
                t = result.AddAritmethicOperator("+", TypeFine.AritmethicOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            readNext();
            t = result.AddXCrementOperator("++", TypeFine.Increment, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }
        private bool handleMinus() // - -- -= ->
        {
            if ('-' != c0) return false;
            Token t;
            if (cn < 0)
            {
                t = result.AddAritmethicOperator("-", TypeFine.AritmethicOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('=' == c1)
            {
                readNext();
                t = result.AddAssignmentOperator("-=", TypeFine.AssignAritmethic, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('>' == c1)
            {
                readNext();
                t = result.AddScopeOperator("->", TypeFine.ScopeInstanceOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('-' != c1)
            {
                t = result.AddAritmethicOperator("-", TypeFine.AritmethicOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            readNext();
            t = result.AddXCrementOperator("--", TypeFine.Decrement, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }
        private bool handleStrOpen()
        {
            if ('"' != c0 && '\'' != c0 && '`' != c0) return false;
            isStrOpen = true;
            strOpenChar = c0;
            sb.Clear();
            sb.Append(c0);
            return true;
        }
        private bool handleStarPerc() // *  *=  %  %=
        {
            if ('*' != c0 && '%' != c0) return false;
            Token t;
            if (cn < 0 || '=' != c1)
            {
                t = result.AddAritmethicOperator("" + c0, TypeFine.AritmethicOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            t = result.AddAssignmentOperator("" + c0 + c1, TypeFine.AssignAritmethic, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            readNext();
            return true;
        }
        private bool handleDot() // .  .=
        {
            if ('.' != c0) return false;
            Token t;
            if (cn < 0)
            {
                t = result.AddAssignmentOperator(".", TypeFine.StringConcat, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('=' != c1)
            {
                if (ParseHelper.IsNumCharSimple(c1)) return handleNumbers();
                t = result.AddAssignmentOperator(".", TypeFine.StringConcat, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            t = result.AddAssignmentOperator(".=", TypeFine.StringConcat, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            readNext();
            return true;
        }
        private bool handleComSem() // ;  ,  @
        {
            if (';' != c0 && ',' != c0 && '@' != c0) return false;
            Token t;
            if (';' == c0)
                t = result.AddOtherOperator(";", TypeFine.Semicolon, lineNumber, bLevel, cLevel, sLevel);
            else if (',' == c0)
                t = result.AddOtherOperator(",", TypeFine.Comma, lineNumber, bLevel, cLevel, sLevel);
            else
                t = result.AddOtherOperator("@", TypeFine.ErrorControl, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }
        private bool handleVars()
        {
            if ('$' != c0) return false;
            if (cn < 0) throw new IOException("Parse error: The code ends with a $ character");
            Token t;
            string str;
            if ('{' == c1)
            {   // ${
                str = "${";
                while (readNext() && '}' != c1) str += c0;
                if ('}' != c1) throw new IOException("Parse error: The code contains a ${… without a closing } character.");
                // c1 is now '}';
                readNext();
                str += '}';
                t = result.AddOther(str, TypeNormal.Variable, TypeFine.VariableExtend, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if (!ParseHelper.IsSimpleWordStartChar(c1)) return false;
            str = "$";
            while (cn > -1 && ParseHelper.IsSimpleWordStartChar(c1))
            {
                str += c1;
                readNext();
            }
            t = result.AddOther(str, TypeNormal.Variable, TypeFine.Variable, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }
        private bool handleColon() // :  ::
        {
            if (':' != c0) return false;
            Token t;
            if (cn < 0 || ':' != c1)
            {
                t = result.AddLogicalOperator(":", TypeFine.Colon, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            readNext();
            t = result.AddScopeOperator("::", TypeFine.ScopeStaticOperator, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }
        private bool handleHashComment() // #…
        {
            if ('#' != c0) return false;
            Token t;
            if (cn < 0)
            {
                t = result.AddComment("#", TypeNormal.Comment, TypeFine.HashComment, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            sb.Clear();
            sb.Append('#');
            while (cn > -1 && '\n' != c1 && '\r' != c1)
            {
                sb.Append(c1);
                readNext();
            }
            t = result.AddComment(sb.ToString(), TypeNormal.Comment, TypeFine.HashComment, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }
        private bool handleOtherOperators() // ?  ^  ^=
        {
            Token t;
            if ('?' == c0)
            {
                t = result.AddLogicalOperator("?", TypeFine.Questionmark, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('~' == c0)
            {
                t = result.AddOperator("~", TypeNormal.BitwiseOperator, TypeFine.BitwiseOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            if ('^' != c0) return false;
            if (cn < 0 || '=' != c1)
            {
                t = result.AddOperator("^", TypeNormal.BitwiseOperator, TypeFine.BitwiseOperator, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            readNext();
            t = result.AddAssignmentOperator("^=", TypeFine.AssignBitwise, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }
        private bool handleNumbers()
        {
            string str;
            Token t;
            #region  starts with 0-9
            if (ParseHelper.IsNumCharSimple(c0))
            {   // 1  10.1  20E-10
                if (cn < 0)
                {
                    t = result.AddInteger("" + c0, TypeFine.Integer, lineNumber, bLevel, cLevel, sLevel);
                    t.IsAfterLinebreak = afterLF;
                    afterLF = false;
                    return true;
                }
                sb.Clear();
                if ('0' == c0)
                {
                    #region 0x2e Integer hexadecimal
                    if ('x' == c1)
                    {
                        sb.Append("0x");
                        readNext();
                        while (cn > -1 && ParseHelper.IsHexChar(c1)) { sb.Append(c1); readNext(); }
                        t = result.AddInteger(sb.ToString(), TypeFine.IntegerHex, lineNumber, bLevel, cLevel, sLevel);
                        t.IsAfterLinebreak = afterLF;
                        afterLF = false;
                        sb.Clear();
                        return true;
                    }
                    #endregion
                    #region 0b001101 Integer binary
                    if ('b' == c1)
                    {
                        sb.Append("0b");
                        readNext();
                        while (cn > -1 && ('0' == c1 || '1' == c1)) { sb.Append(c1); readNext(); }
                        t = result.AddInteger(sb.ToString(), TypeFine.IntegerBin, lineNumber, bLevel, cLevel, sLevel);
                        t.IsAfterLinebreak = afterLF;
                        afterLF = false;
                        sb.Clear();
                        return true;
                    }
                    #endregion
                }
                sb.Append(c0);
                // read all next 0-9
                while (cn > -1 && ParseHelper.IsNumCharSimple(c1)) { readNext(); sb.Append(c0); }
                if ('.' == c1)
                {
                    readNext();
                    sb.Append(c0);
                    while (cn > -1 && ParseHelper.IsNumCharSimple(c1)) { readNext(); sb.Append(c0); }
                }
                if (cn > -1 && 'e' == char.ToLowerInvariant(c1))
                {
                    readNext();
                    sb.Append(c0);
                    if (cn > -1 && ('+' == c1 || '-' == c1)) { readNext(); sb.Append(c0); }
                    while (cn > -1 && ParseHelper.IsNumCharSimple(c1)) { readNext(); sb.Append(c0); }
                    if ('.' == c1)
                    {
                        readNext();
                        sb.Append(c0);
                        while (cn > -1 && ParseHelper.IsNumCharSimple(c1)) { readNext(); sb.Append(c0); }
                    }
                }
                str = sb.ToString();
                sb.Clear();
                if (ParseHelper.IsOctInteger(str))
                {
                    t = result.AddInteger(str, TypeFine.IntegerOct, lineNumber, bLevel, cLevel, sLevel);
                    t.IsAfterLinebreak = afterLF;
                    afterLF = false;
                    return true;
                }
                if (str.IndexOf("e", StringComparison.InvariantCultureIgnoreCase) > -1)
                {
                    t = result.AddFloat(str, TypeFine.FloatExp, lineNumber, bLevel, cLevel, sLevel);
                    t.IsAfterLinebreak = afterLF;
                    afterLF = false;
                    return true;
                }
                if (str.IndexOf('.') > -1)
                {
                    t = result.AddFloat(str, TypeFine.Float, lineNumber, bLevel, cLevel, sLevel);
                    t.IsAfterLinebreak = afterLF;
                    afterLF = false;
                    return true;
                }
                t = result.AddInteger(str, TypeFine.Integer, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            #endregion
            if ('.' != c0) return false;
            if (cn < 0 || ParseHelper.IsNumCharSimple(c1))
            {
                t = result.AddAssignmentOperator(".", TypeFine.StringConcat, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            sb.Clear();
            sb.Append('.');
            while (cn > -1 || ParseHelper.IsNumCharSimple(c1))
            {
                readNext();
                sb.Append(c0);
            }
            if (cn > -1 && 'e' == char.ToLowerInvariant(c1))
            {
                readNext();
                sb.Append(c0);
                if (cn > -1 && ('+' == c1 || '-' == c1)) { readNext(); sb.Append(c0); }
                while (cn > -1 && ParseHelper.IsNumCharSimple(c1)) { readNext(); sb.Append(c0); }
                if ('.' == c1)
                {
                    readNext();
                    sb.Append(c0);
                    while (cn > -1 && ParseHelper.IsNumCharSimple(c1)) { readNext(); sb.Append(c0); }
                }
            }
            str = sb.ToString();
            sb.Clear();
            if (str.IndexOf("e", StringComparison.InvariantCultureIgnoreCase) > -1)
            {
                t = result.AddFloat(str, TypeFine.FloatExp, lineNumber, bLevel, cLevel, sLevel);
                t.IsAfterLinebreak = afterLF;
                afterLF = false;
                return true;
            }
            t = result.AddFloat(str, TypeFine.Float, lineNumber, bLevel, cLevel, sLevel);
            t.IsAfterLinebreak = afterLF;
            afterLF = false;
            return true;
        }

        #endregion

    }

}